# -*- coding:utf-8 -*-
from database.db_handler import MysqlHander
from database.db_business import DbBusiness
from common.my_http import MyHttp
from common.my_baidu import MyBaidu
from common.my_time import MyTime
from common.market_data import MarketData
import urllib.request
from bs4 import BeautifulSoup
import glob
import jieba
import re
import threading
import hashlib
import urllib.parse
import time
import os


'''
网络数据
'''


class BaiduGanhan:
    def __init__(self):
        self.market = MarketData()
        self.business = DbBusiness()
    
    def get_data(self):
        end_date = MyTime.forward_relative_date(5)
        diqu = self.business.query_diqu()
        check_title2 = ["干旱","旱灾"]
        check_title3 = ["水库","减产","影响","灾难","死亡","产量"]
        for d in diqu:
            next_page = True
            words = [d, "干旱"]
            for i in range(1,5):
                time.sleep(1)
                if next_page == False:
                    break
                url = MyBaidu.get_url_order_by_time(words, i)
                try:
                    res = []
                    proxy = self.business.query_proxy()
                    if len(proxy) > 0:
                        res = MyBaidu.get_baidu_data_by_proxy(url, proxy)
                    else:
                        res = MyBaidu.get_baidu_data(url)
                    for r in res:
                        data_date = MyBaidu.calc_date(r["date"])
                        if data_date < end_date:
                            next_page = False
                            break
                        if MyBaidu.check_site(r["site"]) == False:
                            continue
                        if MyBaidu.check_title(r["title"], [[d], check_title2, check_title3]) == False:
                            continue
                        data_title = r["title"]
                        data_site = r["site"]
                        data_url = r["url"]
                        print(r["date"] + data_date)
                        self.market.add_market_data(data_date, 2150, data_title, data_url, data_site)
                except Exception as e:
                    i = i - 1
                    pass

def ganhan_api():
    d = BaiduGanhan()
    d.get_data()


if __name__ == '__main__':
    ganhan_api()

